/**
 *	@author: Kathleen Go
 *	date created: April 23, 2006
 *	@ver: 1.0
 * 	modified: May 3, 2006
 */

package text.util;

import java.util.Vector;
import java.util.Hashtable;
import java.util.Enumeration;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.StringTokenizer;

import text.obj.TExtChunk;
import text.obj.TExtTemplate;
import text.learning.TExtChunkAligner;

public class TExtLearnerUtility{
	
	private boolean bSourceIsFilipino;
	private Vector vFunction1;
	private Vector vFunction2;
	private Vector vFcnAffix1;
	private Vector vFcnAffix2;
	
	private Vector vCommon1;	//1 - Filipino
	private Vector vCommon2;	//2 - English
		
	public TExtLearnerUtility(boolean bSourceIsFilipino1){
		bSourceIsFilipino = bSourceIsFilipino1;
		
		Vector vDummy1 = initializeFcnWords("Filipino");
		Vector vDummy2 = initializeFcnWords("English");
		
		vFunction1 = (Vector)vDummy1.get(0);
		vFcnAffix1 = (Vector)vDummy1.get(1);
		
		vFunction2 = (Vector)vDummy2.get(0);
		vFcnAffix2 = (Vector)vDummy2.get(1);
		
		vCommon1 = initializeCommonWords("Filipino");
		vCommon2 = initializeCommonWords("English");
	}
	
	public Hashtable combineHash(Hashtable htCombined, Hashtable ht2){
		Enumeration keys = ht2.keys();

		while(keys.hasMoreElements()){
			Object oKey = keys.nextElement();
			Vector vValues = (Vector)ht2.get(oKey);
			if(htCombined.containsKey(oKey)){
				Vector vValuesExist = (Vector)htCombined.remove(oKey);
				for(Iterator it = vValues.iterator(); it.hasNext(); ){
					Object oValue = it.next();
					if(!vValuesExist.contains(oValue))
						vValuesExist.add(oValue);
				}
				htCombined.put(oKey, vValuesExist);
			}
			else
				htCombined.put(oKey, ht2.get(oKey));
		}
		return htCombined;
	}
	
	/**	This returns the function words taken from the Database.
	 *	The first element contains the function words applicable to Filipino.
	 *	The second element contains the function words applicable to English.
	 */
	public Vector getFunctionWords(){
		Vector vFunction = new Vector(2);
		vFunction.add(vFunction1);
		vFunction.add(vFunction2);
		return vFunction;	
	}
	
	/**	This function gets the alignment from each Vector (vPossRes1 and vPossRes2) such
	 *		that the splitted chunks from the original identified Differences and the
	 *		number of resulting chunks from each split are the same for the Source chunks
	 *		of the two sets.  This also applies to the Target chunks of both sets.  The
	 *		order of alignment must also be the same such that the indices from the Source
	 *		must be aligned to same indices in the Target for both sets.
	 *	Returns the Vector containing the chosen alignments from vPossRes1 and vPossRes2.
	 *		Index 0 contains chosen alignment from vPossRes1.
	 *		Index 1 contains chosen alignment from vPossRes2.
	 */
	public Vector chooseAmongAlSplit(Vector vPossRes1, Vector vPossRes2){
		Vector vChosen = null;
		boolean bStop = false;
		for(Iterator it1 = vPossRes1.iterator(); it1.hasNext() && !bStop; ){
			Vector vCand1 = (Vector)it1.next();
			for(Iterator it2 = vPossRes2.iterator(); it2.hasNext() && !bStop; ){
				Vector vCand2 = (Vector)it2.next();
				if(((String)vCand2.get(0)).compareTo((String)vCand1.get(0)) == 0){
					vChosen = new Vector(2);
					vChosen.add(vCand1.clone());
					vChosen.add(vCand2.clone());
					bStop = true;
				}
			}
		}
		return vChosen;
	}
	
	/**	Checks if strWord is a punctuation mark.
	 */
	public boolean isPunctuation(String strWord){
		
		if(strWord.equals(",") || strWord.equals("!") || strWord.equals(";") || strWord.equals("(")
			 || strWord.equals(")") || strWord.equals("{") || strWord.equals("}") || strWord.equals("[")
			  || strWord.equals("]") || strWord.equals("\"") || strWord.equals("<") || strWord.equals(">")
			   || strWord.equals("?") || strWord.equals("/") || strWord.equals(".") || strWord.equals("..."))
			   	return true;
		return false;
	}

	public boolean isValidSimilarities(Vector vSource, Vector vTarget, Hashtable alignedWords){
		for(int i = vSource.size()-1; i >= 0; i--)
			if(isPunctuation(vSource.get(i).toString()))
				vSource.remove(i);
		for(int i = vTarget.size()-1; i >= 0; i--)
			if(isPunctuation(vTarget.get(i).toString()))
				vTarget.remove(i);
		if(vSource.size() == 0 || vTarget.size() == 0)
			return false;
					
		TExtChunk cSource = new TExtChunk(TExtUtility.vecToString(vSource), "[0]", makeCompoundIdentifier(vSource));
		cSource = adjustChunkCompound(cSource);
		TExtChunk cTarget = new TExtChunk(TExtUtility.vecToString(vTarget), "[0]", makeCompoundIdentifier(vTarget));
		cTarget = adjustChunkCompound(cTarget);
		Vector vTargetChunks = new Vector(1);
		vTargetChunks.add(cTarget);
		TExtChunkAligner txtChunkAlign = new TExtChunkAligner(bSourceIsFilipino, getFunctionWords(), alignedWords);
		Vector vDummy = txtChunkAlign.alignFromAlignedWords(cSource, vTargetChunks, 'a', true);
		Vector vSourcePOVMap = (Vector)((Vector)vDummy.get(2)).get(0);
		vSource = null;
		vTarget = null;
		txtChunkAlign = null;
		vDummy = null;
		for(Iterator it = vSourcePOVMap.iterator(); it.hasNext(); ){
			int nTargetPos = ((Integer)it.next()).intValue();
			if(nTargetPos != -999 && nTargetPos != -1)
				return true;
		}
		return false;
	}

	/**	This function adjusts the the template whose Difference chunks are not yet
	 *		replaced with variables (strTemp).  Given the Vector of splitted Chunks (vChunks)
	 *		, this function gets the identified Differences from the template representation
	 *		and then divides the said Difference according to how it is seen in vChunks.
	 *	Returns the modified template reprentation.
	 */ 
	public Vector adjustChunkDiv(String strTemp, Vector vChunks, String strCompID){
		StringBuffer sbChunk = new StringBuffer();
		boolean bChunkScan = false;
		String strChunkTemp;
		int nOpen = 0;
		int nClose = 0;
		
		int nRanges = -1;
		String[] strCompIDRanges = null;
		if(strCompID.compareTo("none") != 0){
			strCompIDRanges = strCompID.split(";");
			nRanges = strCompIDRanges.length;
		}
		int nIndex = 0;
		boolean bModCompID = false;
		
		StringBuffer sbTemplate = new StringBuffer();
		int nTempLength = strTemp.length();
		for(int p=0; p<nTempLength; p++) {
			
			if(!bChunkScan){
				char cTemp = strTemp.charAt(p);
				sbTemplate.append(cTemp);
				if(cTemp == ' ')
					nIndex++;
			}
			
			else
				sbChunk.append(strTemp.charAt(p));
			if(strTemp.charAt(p) == '[')
				nOpen++;
			if(strTemp.charAt(p)=='[' && !bChunkScan) {
				sbChunk = new StringBuffer();
				sbChunk.append(strTemp.charAt(p));
				sbTemplate = sbTemplate.deleteCharAt(sbTemplate.length()-1);
				bChunkScan = true;
			} else if(strTemp.charAt(p)==']') {
				nClose ++;
				if(nOpen == nClose){
					StringBuffer sbFromCVec = new StringBuffer("");
					int nEnd = -1;
					int nChunks = vChunks.size();
					for(int i = 0; i < nChunks; i++){
						sbFromCVec.append(((TExtChunk)vChunks.get(i)).toString());
						String strChunk = "[" + sbFromCVec.toString() + "]";
						if(strChunk.compareTo(sbChunk.toString()) == 0){
							nEnd = i;
							break;
						}
						else
							sbFromCVec.append(" ");
					}
					for(int i = 0; i <= nEnd; i++){
						sbTemplate.append("[" + ((TExtChunk)vChunks.get(i)).toString() + "]");
						if(i != nEnd)
							sbTemplate.append(" ");
					}
					for(int i = nEnd; i >= 0; i--)
						vChunks.remove(i);
					
					if(nEnd > 0 && strCompID.compareTo("none") != 0){
						bModCompID = true;
						for(int j = 0; j < nRanges; j++){
							String strRange[] = strCompIDRanges[j].split("-");
							int nLBound = Integer.parseInt(strRange[0]);
							if(nLBound > nIndex){
								int nUBound = Integer.parseInt(strRange[1]);
								nLBound += nEnd;
								nUBound += nEnd;
								strCompIDRanges[j] = new String(nLBound + "-" + nUBound);
							}
							strRange = null;
						}
						nIndex += nEnd;
					}
					
					nOpen = 0;
					nClose = 0;
					bChunkScan = false;
				}
			}
		}
		Vector vResults = new Vector(2);
		if(strCompID.compareTo("none") != 0){
			if(bModCompID){
				StringBuffer sbCompID = new StringBuffer("");
				for(int j = 0; j < nRanges; j++){
					if(j > 0)
						sbCompID.append(";");
					sbCompID.append(strCompIDRanges[j]);
				}
				vResults.add(sbCompID.toString());
			}
			else
				vResults.add(strCompID);
			
		}
		vResults.add(sbTemplate);
		strChunkTemp = null;
		strCompIDRanges = null;
		strTemp = null;
		vChunks = null;
		return vResults;
	}

	/**	This returns the affix labels of the function words taken from the Database.
	 *	The first element contains the Filipino function words that are prefixes.
	 *	The second element contains the English function words that are prefixes.
	 */
	public Vector getFunctionWordsPrefix(){
		Vector vFunctionAffix = new Vector(2);
		vFunctionAffix.add(vFcnAffix1);
		vFunctionAffix.add(vFcnAffix2);
		return vFunctionAffix;	
	}

	/**	This counts the number of non-common words among the tokens contained in vConstants.
	 */
	public int computeSpecificity(Vector vConstants,boolean isSource){
		int nCtr = 0;
		Vector vCommon;
		
		if(isSource)
			if(bSourceIsFilipino)
				vCommon = vCommon1;
			else
				vCommon = vCommon2;
		else
			if(bSourceIsFilipino)
				vCommon = vCommon2;
			else
				vCommon = vCommon1;

		for(Iterator itTemp = vConstants.iterator(); itTemp.hasNext(); ){
			Object token = itTemp.next();
			if(token instanceof String && !vCommon.contains(token.toString().toLowerCase()))
				nCtr++;
		}
		
		return nCtr;
	}

/**	Gets from the DB the function words for the given language (strLanguage).
 */
	public Vector initializeFcnWords(String strLanguage){
		String strQuery = "Select * from noisewords where Language = '"+ strLanguage + "'";
		Vector vFcnLookUp = new Vector();
		Vector vAffixLookUp = new Vector();
	
		ResultSet rsEquivWords;
		
		rsEquivWords = DBConnector.select(strQuery);
		
		try{
			while(rsEquivWords.next()){
				String strWord = rsEquivWords.getString("noiseWord");
				vFcnLookUp.add(strWord);
				if(rsEquivWords.getString("position").compareToIgnoreCase("before") == 0)
					vAffixLookUp.add(strWord);
			}
		}	
		catch(SQLException e){}

		Vector vResults = new Vector(2);
		vResults.add(vFcnLookUp);
		vResults.add(vAffixLookUp);
		return vResults;
	}
	
	public Vector initializeCommonWords(String strLanguage){
		String strQuery = "Select * from commonWords where Language = '"+ strLanguage + "'";
		Vector vCommon = new Vector();
	
		ResultSet rsEquivWords;
		
		rsEquivWords = DBConnector.select(strQuery);
		
		try{
			while(rsEquivWords.next()){
				String strWord = rsEquivWords.getString("commonWord");
				vCommon.add(strWord);
			}
		}	
		catch(SQLException e){}

		return vCommon;
	}

/*
 *Given the TExtTemplate:
 *
 *	I [1] ice cream --> with 4 tokens [I, ate, ice, cream]
 *
 *Return the TExtTemplate:
 *
 *	I [1] ice cream --> with 3 tokens [I, ate, ice cream]
 *
 *by using the strCompID in the TExtTemplate object.
 *
 *How to use the function:
 *	1.) construct a new TExtTemplate
 *		TExtTemplate templateTemp = new TExtTemplate(strTemplate, strCompID);
 *	2.) adjust the newly created TExtTemplate
 *		templateTemp = adjustTemplateCompound(templateTemp);
 *
 *When to use: construct TExtTemplates from the retrieved entries in the Templates DB
 */
 	/**	Given a Template whose tokens are space delimited, this function uses the
 	 *		compound identifier of the Template so that compound words will be taken
 	 *		as one token.
 	 */
	public TExtTemplate adjustTemplateCompound(TExtTemplate tTemp){
		String strCompID = tTemp.getCompoundIdentifier();
		
		if(!strCompID.equals("none")){
			String[] strIndices = strCompID.split(";");
			int nCombine = strIndices.length;
			int nLBound = -1;
			int nUBound = -1;
			for(int i = nCombine - 1; i >= 0; i--){
				StringBuffer sbCompound = new StringBuffer();
				String[] strRange = strIndices[i].split("-");
				try{
					nLBound = Integer.parseInt(strRange[0]);
					nUBound = Integer.parseInt(strRange[1]);
				}
				catch(Exception e){
					
				}
				for(int j = nLBound; j <= nUBound; j++){
					if(j > nLBound)
						sbCompound.append(" ");
					sbCompound.append(tTemp.get(j));
				}
				for(int j = nUBound; j >= nLBound+1; j--)
					tTemp.remove(j);
				if(sbCompound.length() == 0)
					sbCompound.append("none");
				tTemp.set(nLBound, sbCompound.toString());
				strRange = null;
			}
			strIndices = null;
		}
		
		return tTemp;
	}	

/*(Example Case: Template Refinement)
 *Given the Source TExtTemplate from the input sentence:
 *
 *	The ice cream and the ice water are in the [1] --> with 9 tokens [The, ice cream, and, the, ice water, are, in, the, [1]]
 *
 *and the value of its strCompID is:
 *
 *	1-2;5-6
 *
 *For example the following chunk is to be learned:
 *
 *	ice cream --> with 1 token [ice cream]
 *
 *The new template will be:
 *
 *	The [2] and the ice water are in the [1] --> with 9 tokens [The, [2], and, the, ice water, are, in, the, [1]]
 *
 *Since the token "ice cream" was removed and made into a TExtChunk, 
 *	there is a need to edit the strCompID of the given TExtTemplate to:
 *
 *	4-5
 *
 *When to use: when 1 or more tokens are removed from the given Sentence/TExtTemplate/TExtChunk
 */

	/**	Given a compound identifier (strCompoundIdentifier) and the Vector of the indices
	 *		of removed compound words (vRemoved), the ranges (indices) of the
	 *		compound identifier will be modified according to vRemoved.
	 *	Returns the modified strCompoundIdentifier.
	 */
	public String adjustCompIDVal(String strCompoundIdentifier, Vector vRemoved){
		if(strCompoundIdentifier.compareTo("none") == 0)
			return strCompoundIdentifier;
			
		String[] strRemaining = strCompoundIdentifier.split(";");
		int nRemaining = strRemaining.length;
		String strRemoved;
		String[] strRange;

		int nRemoved = vRemoved.size();
		for(int i = 0; i < nRemoved; i++){
			strRemoved = (String)vRemoved.get(i);
			strRange = strRemoved.split("-");
			try{
				int nOffset = Integer.parseInt(strRange[1]) - Integer.parseInt(strRange[0]);
				for(int j = 0; j < nRemaining; j++){
					String[] strTemp = strRemaining[j].split("-");
					int nLBound = -1;
					int nUBound = -1;
					try{
						nLBound = Integer.parseInt(strTemp[0]);
						nUBound = Integer.parseInt(strTemp[1]);
					}
					catch(Exception e){
						
					}
					if(Integer.parseInt(strRange[1]) < nLBound){
						nLBound -= nOffset;
						nUBound -= nOffset;
						strRemaining[j] = nLBound + "-" + nUBound;
					}
				}
				for(int j = i + 1; j < nRemoved; j++){
					String[] strTemp = ((String)vRemoved.get(j)).split("-");
					int nLBound = -1;
					int nUBound = -1;
					try{
						nLBound = Integer.parseInt(strTemp[0]);
						nUBound = Integer.parseInt(strTemp[1]);
					}
					catch(Exception e){
						
					}
					nLBound -= nOffset;
					nUBound -= nOffset;
					vRemoved.setElementAt(nLBound + "-" + nUBound, j);
				}				
			}
			catch(Exception e){
				
			}
		}
		StringBuffer sbCompID = new StringBuffer();
		for(int i = 0; i < nRemaining; i++){
			if(i > 0)
				sbCompID.append(";");
			sbCompID.append(strRemaining[i]);
		}
		if(sbCompID.length() == 0)
			sbCompID.append("none");
		
		strRemaining = null;
		strRemoved = null;
		strRange = null;
		vRemoved = null;

		return sbCompID.toString();
	}

/*Given a Vector of tokens:
 *
 *	the ice cream and the ice water --> with 5 tokens [the, ice cream, and, the, ice water]
 *
 *This function will return the strCompID:
 *
 *	1-2;5-6
 *
 *When to use: when creating a new TExtChunk or TExtTemplate given the Vector of tokens
 */
 	/**	Given a Vector of tokens wherein compound words are taken as 1 token, this
 	 *		function returns the corresponding compound identifier.
 	 */
	public String makeCompoundIdentifier(Vector vChunkTokens){
		StringBuffer sbCompoundField = new StringBuffer();
		Object oToken;
		int nUBound;
		int nLBound = 0;
		
		for(Iterator it = vChunkTokens.iterator(); it.hasNext(); ){
			oToken = it.next();
			if(oToken instanceof String && oToken.toString().contains(" ")){
				nUBound = nLBound + (new StringTokenizer(oToken.toString())).countTokens() - 1;
				if(sbCompoundField.length() > 0)
					sbCompoundField.append(";");
				sbCompoundField.append(nLBound + "-" + nUBound);
				nLBound = nUBound;
			}
			nLBound++;
		}
		if(sbCompoundField.length() == 0)
			sbCompoundField.append("none");
		oToken = null;
		return sbCompoundField.toString();
	}

	/**	This is used when deriving templates from chunks.
	 *	vRemoved contains the the ranges of indices of the removed chunks.
	 *	This function returns the ranges of indices of those compound words that are
	 *		included in the removed chunks.
	 */
	public Vector editCompID(String strTempSCompID, Vector vRemoved){
		Vector vTempRemove = new Vector(vRemoved.size());
		String[] strCompIDRange = strTempSCompID.split(";");
		int nCompIDRange = strCompIDRange.length;
		int nRemoved = vRemoved.size();
		if(strTempSCompID.compareTo("none") == 0)
			return vRemoved;
		
		String[] strIndexOuter;
		
		for(int i = 0; i < nCompIDRange; i++){
			strIndexOuter = strCompIDRange[i].split("-");
			int nLBOuter = Integer.parseInt(strIndexOuter[0]);
			int nUBOuter = Integer.parseInt(strIndexOuter[1]);
			
			for(int j = 0; j < nRemoved; j++){
				String[] strIndexInner = ((String)vRemoved.get(j)).split("-");
				int nLBInner = Integer.parseInt(strIndexInner[0]);
				int nUBInner = Integer.parseInt(strIndexInner[1]);
				
				if(nLBOuter >= nLBInner && nUBOuter <= nUBInner){
					vTempRemove.add(nLBOuter + "-" + nUBOuter);
					break;
				}
			}
		}
		strCompIDRange = null;
		strIndexOuter = null;
		return vTempRemove;
	}
	
	/**	This returns the new compound identifier (strTempSCompID) after the ranges of indices
	 *		(contained in vRemoved) are removed from strTempSCompID
	 */
	public String removeCompID(String strTempSCompID, Vector vRemoved){
		if(strTempSCompID.compareTo("none") == 0)
			return strTempSCompID;
			
		StringBuffer sbNewCompID = new StringBuffer();
		String[] strCompID = strTempSCompID.split(";");
		int nCompID = strCompID.length;

		for(int i = 0; i < nCompID; i++){
			boolean bPresent = false;
			for(int j = 0; j < vRemoved.size(); j++){
				String strRangeInner = (String)vRemoved.get(j);
				if(strCompID[i].compareTo(strRangeInner) == 0){
					vRemoved.remove(j);
					bPresent = true;
				}
			}
			if(!bPresent)
				sbNewCompID = sbNewCompID.append(strCompID[i] + ";");
				
		}
		strCompID = null;
		if(sbNewCompID.length() == 0)
			return "none";
		else
			return sbNewCompID.substring(0, sbNewCompID.length()-1);
	}
	
	/**	This returns the next useable domain for a chunk based on the Chunks DB.
	 */
	public String returnNextChunk() {
		String strNextChunk = null;
		
		java.sql.Statement stmtChunk = DBConnector.createStatement(
			ResultSet.TYPE_SCROLL_INSENSITIVE,
			ResultSet.CONCUR_UPDATABLE);

		try {
			ResultSet rsChunkMax = stmtChunk.executeQuery(
				"Select Max(domain) as maxdomain From chunks");
			if(rsChunkMax.next()) {
				strNextChunk = rsChunkMax.getString("maxdomain");
				if(strNextChunk == null)
					strNextChunk = "1";
				else{
					java.util.StringTokenizer st = new 
						java.util.StringTokenizer(strNextChunk,".");
					strNextChunk = ""+(Integer.parseInt(st.nextToken())+1);
				}
			} else
				strNextChunk = "1";
			rsChunkMax = null;
		} catch(SQLException sqle) {
			System.err.println("Chunk Result Set error:"+sqle.getMessage());
		}
			
		return strNextChunk;
	}
	
/*
 *Given the TExtChunk:
 *
 *	ice cream --> with 2 tokens [ice, cream]
 *
 *Return the TExtTemplate:
 *	
 *	ice cream. --> with 1 token [ice cream]
 *
 *by using the strCompID in the TExtChunk object.
 *
 *How to use the function:
 *	1.) construct a new TExtChunk
 *		TExtChunk chunkTemp = new TExtChunk(strChunk, strDomain, strCompID);
 *	2.) adjust the newly created TExtChunk
 *		chunkTemp = adjustChunkCompound(chunkTemp);
 *
 *When to use: construct TExtChunks from the retrieved entries in the Chunks DB
 */

 	/**	Given a Chunk whose tokens are space delimited, this function uses the
 	 *		compound identifier of the Chunk so that compound words will be taken
 	 *		as one token.
 	 */
	public TExtChunk adjustChunkCompound(TExtChunk cChunk){
		String strCompID = cChunk.getCompoundIdentifier();
				
		if(!strCompID.equals("none")){
			String[] strIndices = strCompID.split(";");
			int nCombine = strIndices.length;
			int nLBound = -1;
			int nUBound = -1;
			for(int i = nCombine - 1; i >= 0; i--){
				StringBuffer sbCompound = new StringBuffer();
				String[] strRange = strIndices[i].split("-");
				try{
					nLBound = Integer.parseInt(strRange[0]);
					nUBound = Integer.parseInt(strRange[1]);
				}
				catch(Exception e){
					
				}
				for(int j = nLBound; j <= nUBound; j++){
					if(j > nLBound)
						sbCompound.append(" ");
					sbCompound.append(cChunk.get(j));
				}
				for(int j = nUBound; j >= nLBound+1; j--)
					cChunk.remove(j);
				cChunk.set(nLBound, sbCompound.toString());
				strRange = null;
			}
			strIndices = null;
		}
		
		return cChunk;
	}

	/**	Converts a chunk (represented as a Vector of tokens) to String.
	 */
	public String toStringChunk(Vector vChunkTokens){
		StringBuffer sbChunk = new StringBuffer();
		
		for(Iterator it = vChunkTokens.iterator(); it.hasNext(); )
			sbChunk = sbChunk.append((it.next()).toString() + " ");
		
		if(sbChunk.length() != 0)
			return sbChunk.substring(0, sbChunk.length()-1);
		else
			return sbChunk.toString();
	}
	
	
	
	
	
	/**
	 * removes duplicates: 99.9/99.9
	 * 
	 * based from generalizeDomainsRefiner
	 */
	public static String removeDuplicateDomains(String strCTValue){
		String strCTValue2 = strCTValue;
		String str = strCTValue;
		boolean bProceed = str.contains("[");
		Vector vAlreadyAdded = new Vector(1);
		
		while( bProceed){
			int nBegin = str.indexOf('[');
			int nEnd = str.indexOf(']');
			String strDomain = str.substring(nBegin+1, nEnd); // 99/22.1/99.1
			String strNewDomain = new String();
			String arrDomains[] = strDomain.split("/");
			String strCurrent = new String();
			boolean bAddDecimals = true;
			vAlreadyAdded = new Vector(1);
			for(int i=0; i < arrDomains.length; i++) {

					strCurrent = arrDomains[i];
					if(!vAlreadyAdded.contains(strCurrent)){
						if(strNewDomain.equals("")){
							strNewDomain+=strCurrent;
						}else{
							strNewDomain+="/"+strCurrent;
						}		
						vAlreadyAdded.add(strCurrent);					
					}
								
			}//end for
			if (!strNewDomain.equals(""))
			strCTValue2 = strCTValue2.replace(strDomain, strNewDomain);
				
			str = str.substring(nEnd+1, str.length());
			bProceed = str.contains("[");
			
		}
		return strCTValue2;
	}
	
	
	
	
	
	
	/**
	 * removes duplicates: 99.9/99.9
	 * and [1/1.6/5.8/3.2] to [1/5.8/3.2]
	 * taken from ChunkRefiner
	 */
	public static String generalizeDomains(String strCTValue){
		String strCTValue2 = strCTValue;
		String str = strCTValue;
		boolean bProceed = str.contains("[");
		Vector vAlreadyAdded = new Vector(1);
		
		while( bProceed){
			int nBegin = str.indexOf('[');
			int nEnd = str.indexOf(']');
			String strDomain = str.substring(nBegin+1, nEnd); // 99/22.1/99.1
			String strNewDomain = new String();
			String arrDomains[] = strDomain.split("/");
			String strCurrentWhole = new String();
			boolean bAddDecimals = true;
			vAlreadyAdded = new Vector(1);
			for(int i=0; i < arrDomains.length; i++) {

				if (!arrDomains[i].contains(".")) {
					strCurrentWhole = arrDomains[i];
					
					ResultSet rsDom = DBConnector.select("Select * From chunks "
							+ "Where domain Like '" + strCurrentWhole + ".%'");
						
						try{
							if(rsDom.next()){
								if(strNewDomain.equals("")){
									strNewDomain+=strCurrentWhole;
									
								}else{
									strNewDomain+="/"+strCurrentWhole;
								}
								bAddDecimals=false;
								vAlreadyAdded.add(strCurrentWhole);
							}else{
								bAddDecimals=true;
							}
							rsDom.getStatement().close();
							rsDom.close();
						}catch(SQLException e){					
						}catch(Exception e){					
						}
				}else{
					//means there is decimal
					boolean bDecimalNotFromWhole = false;
					bDecimalNotFromWhole = !(arrDomains[i].contains(".") 
						&& arrDomains[i].substring(0,arrDomains[i].indexOf('.')).equals(strCurrentWhole) );
						
					if(bAddDecimals){
						
						if(!vAlreadyAdded.contains(arrDomains[i]))
						if(strNewDomain.equals("")){
							strNewDomain+=arrDomains[i];
						}else{
							strNewDomain+="/"+arrDomains[i];
						}
						
						vAlreadyAdded.add(arrDomains[i]);
					}else if (bDecimalNotFromWhole){
						
						if(!vAlreadyAdded.contains(arrDomains[i]))
						if(strNewDomain.equals("")){
							strNewDomain+=arrDomains[i];
						}else{
							strNewDomain+="/"+arrDomains[i];
						}		
						
						vAlreadyAdded.add(arrDomains[i]);
						
					}
					
					
				}//endif
			}//end for
			if (!strNewDomain.equals(""))
			strCTValue2 = strCTValue2.replace(strDomain, strNewDomain);
				
			str = str.substring(nEnd+1, str.length());
			bProceed = str.contains("[");
			
		}
		return strCTValue2;
	}
	
	

}